{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color='green'> 1. Description<font>\n",
    "\n",
    "Sentiment classification using Rotten Tomatoes (Movie) review dataset (binary classification).\n",
    "Dataset can be downloaded from https://drive.google.com/file/d/1w1TsJB-gmIkZ28d1j7sf1sqcPmHXw352/view\n",
    "Please dowonload the data manually with your browser and store it to `datasets` directory.\n",
    "    \n",
    "The Rotten Tomatoes movie review dataset is a corpus of movie reviews used for sentiment analysis. We will classify a review to be positive ('fresh') or negative ('rotten') on the basis of review text.\n",
    "Using this dataset, we train a classifier to predict movie rating based on the review text."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color='green'> 2. Data Preprocessing<font>\n",
    "\n",
    "For RT review classification we will perform some data preparation and data cleaning steps. We will generate feature vectors using sklearn TF-IDF for review text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import OrderedDict\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_review(review):\n",
    "    pattern = re.compile(r'[^a-zA-Z0-9 ]')\n",
    "    review = pattern.sub(' ', review)\n",
    "    return review\n",
    "\n",
    "def document_vector_frovedis(doc, frov_w2v_model, frov_vocab):\n",
    "    \"\"\"Create document vectors by averaging word vectors. Remove out-of-vocabulary words.\"\"\"\n",
    "    no_embedding = np.zeros(frov_w2v_model.shape[1])\n",
    "    vocab_doc = [word for word in doc if word in frov_vocab]\n",
    "    if len(vocab_doc) != 0:\n",
    "        return list(np.mean(frov_w2v_model.loc[vocab_doc], axis=0))\n",
    "    else:\n",
    "        return list(no_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_w2v_embed(df):\n",
    "    from frovedis.exrpc.server import FrovedisServer\n",
    "    from frovedis.mllib.feature.w2v import Word2Vec as Frovedis_Word2Vec\n",
    "    os.environ[\"VE_OMP_NUM_THREADS\"] = '8'\n",
    "    FrovedisServer.initialize(\"mpirun -np 1 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "    frovedis_w2v = Frovedis_Word2Vec(sentences = list(df[\"Review\"]), hiddenSize=512, minCount=2, n_iter=100)\n",
    "    X_emb = frovedis_w2v.transform(list(df[\"Review\"]), func = np.mean)\n",
    "    os.environ[\"VE_OMP_NUM_THREADS\"] = '1'\n",
    "    FrovedisServer.shut_down()\n",
    "    return pd.DataFrame(X_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(fname):\n",
    "    df = pd.read_csv(fname, encoding=\"ISO-8859-1\")\n",
    "    df = df.dropna().drop_duplicates().sample(frac=1, random_state=42)\n",
    "    df['Review'] = df['Review'].str.lower().apply(clean_review)\n",
    "    mapping = {'fresh': 1, 'rotten': 0}\n",
    "    df['Freshness'] = df.replace({'Freshness': mapping})\n",
    "    print(\"Dataset contains {} reviews\".format(df.shape[0]))\n",
    "    \n",
    "    stop = stopwords.words('english')\n",
    "    df['Review'] = df['Review'].apply(lambda x: [item for item in word_tokenize(x) if item not in stop])\n",
    "    X = create_w2v_embed(df)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, df[\"Freshness\"], random_state=42)\n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset contains 339716 reviews\n",
      "Generated vector for train data are of shape (254787, 512)\n",
      "Generated vector for test data are of shape (84929, 512)\n"
     ]
    }
   ],
   "source": [
    "#---- Data Preparation ----\n",
    "DATA_FILE = \"datasets/rt_reviews.csv\"\n",
    "X_train, X_test, y_train, y_test = preprocess_data(DATA_FILE)\n",
    "\n",
    "print(\"Generated vector for train data are of shape {}\".format(X_train.shape))\n",
    "print(\"Generated vector for test data are of shape {}\".format(X_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color='green'> 3. Algorithm Evaluation<font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_time = []\n",
    "test_time = []\n",
    "accuracy = []\n",
    "precision = []\n",
    "recall = []\n",
    "f1 = []\n",
    "estimator_name = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(estimator, estimator_nm, \n",
    "             X_train, y_train,\n",
    "             X_test, y_test):\n",
    "    estimator_name.append(estimator_nm)\n",
    "    start_time = time.time()\n",
    "    estimator.fit(X_train, list(y_train))\n",
    "    train_time.append(round(time.time() - start_time, 4))\n",
    "\n",
    "    start_time = time.time()\n",
    "    pred_y = estimator.predict(X_test)\n",
    "    test_time.append(round(time.time() - start_time, 4))\n",
    "\n",
    "    accuracy.append(metrics.accuracy_score(list(y_test), list(pred_y)))\n",
    "    precision.append(metrics.precision_score(list(y_test), list(pred_y), average='macro'))\n",
    "    recall.append(metrics.recall_score(list(y_test), list(pred_y), average='macro'))\n",
    "    f1.append(metrics.f1_score(list(y_test), list(pred_y), average='macro'))\n",
    "\n",
    "    return metrics.classification_report(list(y_test), list(pred_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1 Kernel SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adityaw/virt1/lib64/python3.6/site-packages/sklearn/svm/_base.py:258: ConvergenceWarning: Solver terminated early (max_iter=10000).  Consider pre-processing your data with StandardScaler or MinMaxScaler.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis SVC metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.75      0.76     38123\n",
      "           1       0.80      0.82      0.81     46806\n",
      "\n",
      "    accuracy                           0.79     84929\n",
      "   macro avg       0.79      0.79      0.79     84929\n",
      "weighted avg       0.79      0.79      0.79     84929\n",
      "\n",
      "Sklearn SVC metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.57      0.78      0.66     38123\n",
      "           1       0.75      0.53      0.62     46806\n",
      "\n",
      "    accuracy                           0.64     84929\n",
      "   macro avg       0.66      0.65      0.64     84929\n",
      "weighted avg       0.67      0.64      0.64     84929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: SVC\n",
    "import frovedis\n",
    "target = \"SVC\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "os.environ[\"VE_OMP_NUM_THREADS\"] = \"8\"\n",
    "FrovedisServer.initialize(\"mpirun -np 1 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.svm import SVC as frovSVC\n",
    "\n",
    "f_est = frovSVC(cache_size=2048, max_iter=10000, gamma=1.0)\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "f_est.release()\n",
    "os.environ[\"VE_OMP_NUM_THREADS\"] = \"1\"\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "import sklearn\n",
    "from sklearn.svm import SVC as skSVC\n",
    "s_est = skSVC(cache_size=2048, max_iter=10000, gamma=1.0)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis SVC metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn SVC metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.2 Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis Decision Tree metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.51      0.56     38123\n",
      "           1       0.65      0.74      0.69     46806\n",
      "\n",
      "    accuracy                           0.64     84929\n",
      "   macro avg       0.64      0.63      0.63     84929\n",
      "weighted avg       0.64      0.64      0.63     84929\n",
      "\n",
      "Sklearn Decision Tree metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.52      0.57      0.54     38123\n",
      "           1       0.62      0.58      0.60     46806\n",
      "\n",
      "    accuracy                           0.57     84929\n",
      "   macro avg       0.57      0.57      0.57     84929\n",
      "weighted avg       0.58      0.57      0.58     84929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: DecisionTreeClassifier\n",
    "import frovedis\n",
    "target = \"decision_tree\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "FrovedisServer.initialize(\"mpirun -np 8 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.tree import DecisionTreeClassifier as frovDecisionTreeClassifier\n",
    "\n",
    "f_est = frovDecisionTreeClassifier(max_leaf_nodes=2, max_depth=8)\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "f_est.release()\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "import sklearn\n",
    "from sklearn.tree import DecisionTreeClassifier as skDecisionTreeClassifier\n",
    "s_est = skDecisionTreeClassifier(max_leaf_nodes=2, max_depth=8)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis Decision Tree metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn Decision Tree metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.3 Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis Random Forest metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.54      0.61     38123\n",
      "           1       0.69      0.81      0.74     46806\n",
      "\n",
      "    accuracy                           0.69     84929\n",
      "   macro avg       0.69      0.68      0.68     84929\n",
      "weighted avg       0.69      0.69      0.69     84929\n",
      "\n",
      "Sklearn Random Forest metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.57      0.62     38123\n",
      "           1       0.69      0.80      0.74     46806\n",
      "\n",
      "    accuracy                           0.69     84929\n",
      "   macro avg       0.70      0.68      0.68     84929\n",
      "weighted avg       0.70      0.69      0.69     84929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: RandomForestClassifier\n",
    "import frovedis\n",
    "target = \"random_forest\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "FrovedisServer.initialize(\"mpirun -np 8 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.ensemble import RandomForestClassifier as frovRandomForestClassifier\n",
    "\n",
    "f_est = frovRandomForestClassifier(n_estimators=200, criterion='entropy', max_features=0.5, max_depth=10, \\\n",
    "                                   min_samples_split=5, min_samples_leaf=2)\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "f_est.release()\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier as skRandomForestClassifier\n",
    "s_est = skRandomForestClassifier(n_estimators=200, criterion='entropy', max_features=0.5, max_depth=10, \\\n",
    "                                 min_samples_split=5, min_samples_leaf=2, n_jobs=12)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis Random Forest metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn Random Forest metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.4 Gradient Boosting Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis Gradient Boosting metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.64      0.68     38123\n",
      "           1       0.73      0.80      0.76     46806\n",
      "\n",
      "    accuracy                           0.73     84929\n",
      "   macro avg       0.73      0.72      0.72     84929\n",
      "weighted avg       0.73      0.73      0.73     84929\n",
      "\n",
      "Sklearn Gradient Boosting metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.62      0.67     38123\n",
      "           1       0.72      0.81      0.77     46806\n",
      "\n",
      "    accuracy                           0.73     84929\n",
      "   macro avg       0.73      0.72      0.72     84929\n",
      "weighted avg       0.73      0.73      0.72     84929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: GradientBoostingClassifier\n",
    "\n",
    "target = \"gradient_boosting\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "FrovedisServer.initialize(\"mpirun -np 8 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.ensemble.gbtree import GradientBoostingClassifier as frovGBC\n",
    "\n",
    "f_est = frovGBC(n_estimators=100)\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "f_est.release()\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier as skGBC\n",
    "s_est = skGBC(n_estimators=100)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis Gradient Boosting metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn Gradient Boosting metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.5 Linear SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis Linear SVC metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.93      0.74     38123\n",
      "           1       0.90      0.53      0.67     46806\n",
      "\n",
      "    accuracy                           0.71     84929\n",
      "   macro avg       0.76      0.73      0.70     84929\n",
      "weighted avg       0.77      0.71      0.70     84929\n",
      "\n",
      "Sklearn Linear SVC metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.73      0.74     38123\n",
      "           1       0.79      0.82      0.80     46806\n",
      "\n",
      "    accuracy                           0.78     84929\n",
      "   macro avg       0.77      0.77      0.77     84929\n",
      "weighted avg       0.78      0.78      0.78     84929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: Linear SVC\n",
    "\n",
    "target = \"Linear_SVC\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "FrovedisServer.initialize(\"mpirun -np 8 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.svm import LinearSVC as frovSVC\n",
    "\n",
    "f_est = frovSVC(loss='hinge', max_iter = 60000)\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "f_est.release()\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "from sklearn.svm import LinearSVC as skSVC\n",
    "s_est = skSVC(loss='hinge', max_iter = 60000)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis Linear SVC metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn Linear SVC metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.6 Bernoulli Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis Bernoulli Naive Bayes metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.71      0.70     38123\n",
      "           1       0.76      0.74      0.75     46806\n",
      "\n",
      "    accuracy                           0.73     84929\n",
      "   macro avg       0.72      0.72      0.72     84929\n",
      "weighted avg       0.73      0.73      0.73     84929\n",
      "\n",
      "Sklearn Bernoulli Naive Bayes metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.71      0.70     38123\n",
      "           1       0.76      0.74      0.75     46806\n",
      "\n",
      "    accuracy                           0.73     84929\n",
      "   macro avg       0.72      0.72      0.72     84929\n",
      "weighted avg       0.73      0.73      0.73     84929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: Bernoulli Naive Bayes\n",
    "\n",
    "target = \"bernoulli_naive_bayes\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "FrovedisServer.initialize(\"mpirun -np 8 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.naive_bayes import BernoulliNB as frovNB\n",
    "\n",
    "f_est = frovNB(alpha=1.0)\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "f_est.release()\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "from sklearn.naive_bayes import BernoulliNB as skNB\n",
    "s_est = skNB(alpha=1.0)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis Bernoulli Naive Bayes metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn Bernoulli Naive Bayes metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.7 Nearest Neighbor Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis KNeighborsClassifier metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.55      0.58     22312\n",
      "           1       0.66      0.71      0.69     27688\n",
      "\n",
      "    accuracy                           0.64     50000\n",
      "   macro avg       0.63      0.63      0.63     50000\n",
      "weighted avg       0.64      0.64      0.64     50000\n",
      "\n",
      "Sklearn KNeighborsClassifier metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.55      0.58     22312\n",
      "           1       0.66      0.71      0.69     27688\n",
      "\n",
      "    accuracy                           0.64     50000\n",
      "   macro avg       0.63      0.63      0.63     50000\n",
      "weighted avg       0.64      0.64      0.64     50000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: Nearest Neighbor Classification\n",
    "\n",
    "target = \"nearest_neighbor_classification\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "FrovedisServer.initialize(\"mpirun -np 8 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.neighbors import KNeighborsClassifier as frovKNC\n",
    "\n",
    "f_est = frovKNC(n_neighbors=3, algorithm='brute', metric='euclidean')\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "\n",
    "#we are using first 50k samples inorder to avoid memory issue at Frovedis Server\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train[:50000], y_train[:50000], X_test[:50000], y_test[:50000])\n",
    "f_est.release()\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier as skKNC\n",
    "s_est = skKNC(n_neighbors=3, algorithm='brute', metric='euclidean', n_jobs=12)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train[:50000], y_train[:50000], X_test[:50000], y_test[:50000])\n",
    "\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis KNeighborsClassifier metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn KNeighborsClassifier metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.8 SGDClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frovedis SGDClassifier metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.60      0.94      0.73     38123\n",
      "           1       0.91      0.48      0.63     46806\n",
      "\n",
      "    accuracy                           0.69     84929\n",
      "   macro avg       0.75      0.71      0.68     84929\n",
      "weighted avg       0.77      0.69      0.68     84929\n",
      "\n",
      "Sklearn SGDClassifier metrices: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.72      0.74     38123\n",
      "           1       0.78      0.82      0.80     46806\n",
      "\n",
      "    accuracy                           0.78     84929\n",
      "   macro avg       0.78      0.77      0.77     84929\n",
      "weighted avg       0.78      0.78      0.78     84929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Demo: SGDClassifier\n",
    "\n",
    "target = \"SGDClassifier\"\n",
    "from frovedis.exrpc.server import FrovedisServer\n",
    "FrovedisServer.initialize(\"mpirun -np 8 \" + os.environ[\"FROVEDIS_SERVER\"])\n",
    "from frovedis.mllib.linear_model import SGDClassifier as frovSGD\n",
    "\n",
    "f_est = frovSGD(learning_rate=\"invscaling\", eta0=1.0)\n",
    "e_nm = target + \"_frovedis_\" + frovedis.__version__\n",
    "f_report = evaluate(f_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "f_est.release()\n",
    "FrovedisServer.shut_down()\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier as skSGD\n",
    "s_est = skSGD(learning_rate=\"invscaling\", eta0=1.0, n_jobs=12)\n",
    "e_nm = target + \"_sklearn_\" + sklearn.__version__\n",
    "s_report = evaluate(s_est, e_nm, \\\n",
    "                    X_train, y_train, X_test, y_test)\n",
    "\n",
    "# Precision, Recall and F1 score for each class\n",
    "print(\"Frovedis SGDClassifier metrices: \")\n",
    "print(f_report)\n",
    "print(\"Sklearn SGDClassifier metrices: \")\n",
    "print(s_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color='green'> 4. Performance summary<font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>estimator</th>\n",
       "      <th>train time</th>\n",
       "      <th>test time</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SVC_frovedis_0.9.10</td>\n",
       "      <td>941.4206</td>\n",
       "      <td>315.0987</td>\n",
       "      <td>0.788612</td>\n",
       "      <td>0.786607</td>\n",
       "      <td>0.785453</td>\n",
       "      <td>0.785965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SVC_sklearn_0.24.1</td>\n",
       "      <td>2078.4573</td>\n",
       "      <td>947.4081</td>\n",
       "      <td>0.640041</td>\n",
       "      <td>0.659135</td>\n",
       "      <td>0.652947</td>\n",
       "      <td>0.638762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>decision_tree_frovedis_0.9.10</td>\n",
       "      <td>2.8202</td>\n",
       "      <td>0.2429</td>\n",
       "      <td>0.639499</td>\n",
       "      <td>0.635342</td>\n",
       "      <td>0.627583</td>\n",
       "      <td>0.627328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>decision_tree_sklearn_0.24.1</td>\n",
       "      <td>34.3698</td>\n",
       "      <td>0.1334</td>\n",
       "      <td>0.574303</td>\n",
       "      <td>0.572875</td>\n",
       "      <td>0.573583</td>\n",
       "      <td>0.572458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>random_forest_frovedis_0.9.10</td>\n",
       "      <td>774.0633</td>\n",
       "      <td>24.7382</td>\n",
       "      <td>0.692331</td>\n",
       "      <td>0.694913</td>\n",
       "      <td>0.678575</td>\n",
       "      <td>0.678982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>random_forest_sklearn_0.24.1</td>\n",
       "      <td>1616.4136</td>\n",
       "      <td>0.6313</td>\n",
       "      <td>0.694816</td>\n",
       "      <td>0.695298</td>\n",
       "      <td>0.682880</td>\n",
       "      <td>0.683811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gradient_boosting_frovedis_0.9.10</td>\n",
       "      <td>46.0937</td>\n",
       "      <td>2.9796</td>\n",
       "      <td>0.727419</td>\n",
       "      <td>0.726082</td>\n",
       "      <td>0.719701</td>\n",
       "      <td>0.721224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gradient_boosting_sklearn_0.24.1</td>\n",
       "      <td>4699.1478</td>\n",
       "      <td>0.6603</td>\n",
       "      <td>0.726313</td>\n",
       "      <td>0.726868</td>\n",
       "      <td>0.716526</td>\n",
       "      <td>0.718268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Linear_SVC_frovedis_0.9.10</td>\n",
       "      <td>132.3494</td>\n",
       "      <td>0.1865</td>\n",
       "      <td>0.708392</td>\n",
       "      <td>0.758359</td>\n",
       "      <td>0.728768</td>\n",
       "      <td>0.703778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Linear_SVC_sklearn_0.24.1</td>\n",
       "      <td>95.4360</td>\n",
       "      <td>0.0725</td>\n",
       "      <td>0.776566</td>\n",
       "      <td>0.774965</td>\n",
       "      <td>0.771943</td>\n",
       "      <td>0.773069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>bernoulli_naive_bayes_frovedis_0.9.10</td>\n",
       "      <td>0.4796</td>\n",
       "      <td>0.1802</td>\n",
       "      <td>0.726136</td>\n",
       "      <td>0.723592</td>\n",
       "      <td>0.724743</td>\n",
       "      <td>0.724012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>bernoulli_naive_bayes_sklearn_0.24.1</td>\n",
       "      <td>2.1609</td>\n",
       "      <td>0.7465</td>\n",
       "      <td>0.726136</td>\n",
       "      <td>0.723592</td>\n",
       "      <td>0.724743</td>\n",
       "      <td>0.724012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>nearest_neighbor_classification_frovedis_0.9.10</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>6.6661</td>\n",
       "      <td>0.639420</td>\n",
       "      <td>0.634059</td>\n",
       "      <td>0.630945</td>\n",
       "      <td>0.631494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>nearest_neighbor_classification_sklearn_0.24.1</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>43.5737</td>\n",
       "      <td>0.639420</td>\n",
       "      <td>0.634059</td>\n",
       "      <td>0.630945</td>\n",
       "      <td>0.631494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>SGDClassifier_frovedis_0.9.10</td>\n",
       "      <td>2.6931</td>\n",
       "      <td>0.1828</td>\n",
       "      <td>0.688351</td>\n",
       "      <td>0.753852</td>\n",
       "      <td>0.711891</td>\n",
       "      <td>0.680428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>SGDClassifier_sklearn_0.24.1</td>\n",
       "      <td>3.3646</td>\n",
       "      <td>0.0530</td>\n",
       "      <td>0.776472</td>\n",
       "      <td>0.775058</td>\n",
       "      <td>0.771539</td>\n",
       "      <td>0.772800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          estimator  train time  test time  \\\n",
       "0                               SVC_frovedis_0.9.10    941.4206   315.0987   \n",
       "1                                SVC_sklearn_0.24.1   2078.4573   947.4081   \n",
       "2                     decision_tree_frovedis_0.9.10      2.8202     0.2429   \n",
       "3                      decision_tree_sklearn_0.24.1     34.3698     0.1334   \n",
       "4                     random_forest_frovedis_0.9.10    774.0633    24.7382   \n",
       "5                      random_forest_sklearn_0.24.1   1616.4136     0.6313   \n",
       "6                 gradient_boosting_frovedis_0.9.10     46.0937     2.9796   \n",
       "7                  gradient_boosting_sklearn_0.24.1   4699.1478     0.6603   \n",
       "8                        Linear_SVC_frovedis_0.9.10    132.3494     0.1865   \n",
       "9                         Linear_SVC_sklearn_0.24.1     95.4360     0.0725   \n",
       "10            bernoulli_naive_bayes_frovedis_0.9.10      0.4796     0.1802   \n",
       "11             bernoulli_naive_bayes_sklearn_0.24.1      2.1609     0.7465   \n",
       "12  nearest_neighbor_classification_frovedis_0.9.10      0.1250     6.6661   \n",
       "13   nearest_neighbor_classification_sklearn_0.24.1      0.0385    43.5737   \n",
       "14                    SGDClassifier_frovedis_0.9.10      2.6931     0.1828   \n",
       "15                     SGDClassifier_sklearn_0.24.1      3.3646     0.0530   \n",
       "\n",
       "    accuracy  precision    recall  f1-score  \n",
       "0   0.788612   0.786607  0.785453  0.785965  \n",
       "1   0.640041   0.659135  0.652947  0.638762  \n",
       "2   0.639499   0.635342  0.627583  0.627328  \n",
       "3   0.574303   0.572875  0.573583  0.572458  \n",
       "4   0.692331   0.694913  0.678575  0.678982  \n",
       "5   0.694816   0.695298  0.682880  0.683811  \n",
       "6   0.727419   0.726082  0.719701  0.721224  \n",
       "7   0.726313   0.726868  0.716526  0.718268  \n",
       "8   0.708392   0.758359  0.728768  0.703778  \n",
       "9   0.776566   0.774965  0.771943  0.773069  \n",
       "10  0.726136   0.723592  0.724743  0.724012  \n",
       "11  0.726136   0.723592  0.724743  0.724012  \n",
       "12  0.639420   0.634059  0.630945  0.631494  \n",
       "13  0.639420   0.634059  0.630945  0.631494  \n",
       "14  0.688351   0.753852  0.711891  0.680428  \n",
       "15  0.776472   0.775058  0.771539  0.772800  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ---- evaluation summary ----\n",
    "summary = pd.DataFrame(OrderedDict({ \"estimator\": estimator_name,\n",
    "                                     \"train time\": train_time,\n",
    "                                     \"test time\": test_time,\n",
    "                                     \"accuracy\": accuracy,\n",
    "                                     \"precision\": precision,\n",
    "                                     \"recall\": recall,\n",
    "                                     \"f1-score\": f1\n",
    "                                  }))\n",
    "summary"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
